R Notebook

knitr::opts_chunk$set(warning = FALSE, message = FALSE)
library(tidyverse)
library(lsa)
library(readxl)
library(plotly)
library(countrycode)

df <- read_delim('../../data/country_portfolios_dimensions.csv',delim = ';') %>% 
  rename(N=p) %>% 
  filter(type !='all') %>% 
  mutate(type = case_match(type,
                           'national' ~ 'National',
                           'international' ~ 'International',
                           'migrant' ~ 'Mobile'),
         country_code =countrycode(country_code, origin = 'iso2c', destination = 'country.name')
)

countries_keep <- df %>% group_by(country_code) %>% 
  summarise(N=sum(N)) %>% 
  filter(N>1000) %>% 
  pull(country_code)

df <- df %>% 
  filter(country_code %in% countries_keep)

df_clean <- df %>% 
  group_by(type,for_group_id,country_code) %>% 
  summarise(N = sum(N)) %>% 
  ungroup() %>% 
  mutate(for_group_id = factor(for_group_id),
         country_code = factor(country_code)) %>% 
     complete(type,for_group_id,country_code,fill=list(N=0))

#filter empty topics

empty_topics <- df_clean %>% 
  group_by(for_group_id,country_code) %>% 
  summarise(N= sum(N)) %>% 
  filter(N==0)

df_clean2 <- df_clean %>% 
  anti_join(empty_topics)

df_clean <- df_clean %>% 
  group_by(type,country_code) %>% 
  reframe(for_group_id,
          N,
          p = N/sum(N))

df_clean2 <- df_clean2 %>% 
  group_by(type,country_code) %>% 
  reframe(for_group_id,
          N,
          p = N/sum(N))

cossims <- function(df){
  df %>% 
  select(-country_code,-N) %>% 
  pivot_wider(names_from = type,values_from = p) %>% 
  summarise(cos_IM = cosine(International,Mobile),
            cos_IN = cosine(International,National),
            cos_MN = cosine(Mobile,National))
}

cosine_sim <- df_clean %>% 
  group_by(country_code) %>% 
  cossims()

size versus change

ggplotly(
df %>% group_by(country_code) %>% 
  summarise(N=sum(N)) %>% 
  right_join(cosine_sim) %>% 
  pivot_longer(cols = cos_IM:cos_MN,names_to = 'relation',values_to = 'cosine',names_prefix = 'cos_') %>% 
  mutate(relation = case_when(relation=='IM' ~'International-Mobile',
                              relation=='IN' ~'International-National',
                              relation=='MN' ~'Mobile-National',
                              )) %>% 
  filter(country_code!='ZZALL') %>% 
  ggplot(aes(N, cosine, color=relation, label=country_code)) +
  geom_point()+
  
  scale_x_log10()
)

Try to normalize size

compare all papers vs all papers minus (international/Mobile/National)

cossims_diff <- function(df){
  df %>%
  select(-country_code,-p) %>% 
  pivot_wider(id_cols = c(country_code,for_group_id),names_from = type,values_from = N) %>% 
  mutate(All = International + Mobile + National,
         International = All - International,
         Mobile = All - Mobile,
         National = All - National) %>% 
  mutate(International = International/sum(International),
         Mobile = Mobile/sum(Mobile),
         National = National/sum(National)) %>% 
  summarise(International = cosine(International,All),
            National = cosine(National,All),
            Mobile = cosine(Mobile,All))
}

cosine_sim_diff <- df_clean %>% 
  group_by(country_code) %>% 
  cossims_diff()

ggplotly(
df %>% group_by(country_code) %>% 
  summarise(N=sum(N)) %>% 
  right_join(cosine_sim_diff) %>% 
  pivot_longer(cols = International:Mobile,names_to = 'relation',values_to = 'cosine') %>% 
  filter(country_code!='ZZALL') %>% 
  ggplot(aes(N, cosine, color=relation, label=country_code)) +
  geom_point()+
  
  scale_x_log10()
)

group sizes

ggplotly(
df %>% group_by(country_code, type) %>% 
  summarise(N=sum(N)) %>% 
  right_join(cosine_sim_diff %>% pivot_longer(International:Mobile, names_to='type', values_to='cosine')) %>% 
  # pivot_longer(cols = International:Mobile,names_to = 'relation',values_to = 'cosine') %>% 
  filter(country_code!='ZZALL') %>% 
  group_by(country_code) %>% 
  mutate(country_N = sum(N)) %>% 
  ggplot(aes(country_N, cosine, color=type,size=N, label=country_code)) +
  geom_point()+
  # 
  scale_x_log10()
)

NA

excluding topics that have all 0’s

cosine_sim_diff2 <- df_clean2 %>% 
  group_by(country_code) %>% 
  cossims_diff()

ggplotly(
df %>% group_by(country_code, type) %>% 
  summarise(N=sum(N)) %>% 
  right_join(cosine_sim_diff2 %>% pivot_longer(International:Mobile, names_to='type', values_to='cosine')) %>% 
  # pivot_longer(cols = International:Mobile,names_to = 'relation',values_to = 'cosine') %>% 
  filter(country_code!='ZZALL') %>% 
  group_by(country_code) %>% 
  mutate(country_N = sum(N)) %>% 
  ggplot(aes(country_N, cosine, color=type,size=N, label=country_code)) +
  geom_point()+
  
  scale_x_log10()
)

NA